/*
 * DataSet.java
 * 
 * Copyright 2010-2012 Toolsverse. All rights reserved. Toolsverse
 * PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
 */

package com.toolsverse.etl.common;

import java.io.Serializable;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import javax.script.Bindings;

import com.toolsverse.etl.common.DataSetData.SortType;
import com.toolsverse.etl.driver.Driver;
import com.toolsverse.etl.sql.util.SqlUtils;
import com.toolsverse.util.FileUtils;
import com.toolsverse.util.ListHashMap;
import com.toolsverse.util.Script;
import com.toolsverse.util.TypedKeyValue;
import com.toolsverse.util.Utils;
import com.toolsverse.util.encryption.Base64;

/**
 * <code>DataSet</code> is an in memory table of data representing a database
 * result set. It is usually generated by executing a statement that queries the
 * database or loading from the external source such as comma delimited or xml
 * file.
 * 
 * @author Maksym Sherbinin
 * @version 2.0
 * @since 1.0
 */

public class DataSet implements Serializable
{
    // xml dictionary
    
    /** The DATA_SET attribute. */
    public static final String DATA_SET = "dataset";
    
    /** The NAME attribute. */
    public static final String NAME = "name";
    
    /** The META_DATA attribute. */
    public static final String META_DATA = "metadata";
    
    /** The DATA attribute. */
    public static final String DATA = "data";
    
    /** The ROW attribute. */
    public static final String ROW = "row";
    
    /** The COL attribute. */
    public static final String COL = "col";
    
    /** The VALUE attribute. */
    public static final String VALUE = "value";
    
    /** The TYPE_NAME attribute. */
    public static final String TYPE_NAME = "name";
    
    /** The TYPE_ATTR attribute. */
    public static final String TYPE_ATTR = "type";
    
    /** The NATIVE_TYPE_ATTR attribute. */
    public static final String NATIVE_TYPE_ATTR = "native_type";
    
    /** The NULLABLE_ATTR attribute. */
    public static final String NULLABLE_ATTR = "nullable";
    
    /** The ENCODE_ATTR attribute. */
    public static final String ENCODE_ATTR = "encode";
    
    /** Internal datetime format. */
    public static final String DATA_SET_DATE_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
    
    /** Internal date format. */
    public static final String DATA_SET_DATE_ONLY_FORMAT = "yyyy-MM-dd";
    
    /** Internal time format. */
    public static final String DATA_SET_TIME_FORMAT = "HH:mm:ss";
    
    /** The name. */
    private String _name;
    
    /** The owner name. */
    private String _ownerName;
    
    /** The fields. */
    private ListHashMap<String, FieldDef> _fields;
    
    /** The data. */
    private DataSetData _data;
    
    /** The key name. */
    private String _keyName;
    
    /** The key field. */
    private String _keyFields;
    
    /** The display key field. */
    private String _displayKeyField;
    
    /** The encode. */
    private boolean _encode;
    
    /** The driver. */
    private Driver _driver;
    
    /** The table name. */
    private String _tableName;
    
    /** The variables. */
    private Map<String, Variable> _variables;
    
    /** The has large objects property. */
    private Boolean _hasLargeObj;
    
    /** The functions used to display value in the data set table. */
    private Map<String, TypedKeyValue<String, String>> _displayFunctions;
    
    /** The connection. */
    transient private Connection _connection;
    
    /** The script */
    private Script _script;
    
    /** fields for filter */
    private TreeMap<String, FieldDef> _nonCaseSensitiveFields;
    
    /** filter variables */
    private LinkedHashMap<String, String> _condVars;
    
    /** filter */
    private String _filter;
    
    /** The data set index */
    private Map<String, DataSetRecord> _dataSetIndex;
    
    private Map<String, FieldDef> _indexFields;
    
    /**
     * Instantiates a new data set.
     */
    public DataSet()
    {
        _name = null;
        _ownerName = null;
        _fields = null;
        _data = null;
        _keyName = null;
        _keyFields = null;
        _displayKeyField = null;
        _encode = true;
        _driver = null;
        _tableName = null;
        _connection = null;
        _variables = null;
        _hasLargeObj = null;
        _displayFunctions = null;
        _script = null;
        _nonCaseSensitiveFields = null;
        _condVars = null;
        _dataSetIndex = null;
        _indexFields = null;
        _filter = null;
    }
    
    /**
     * Adds the field.
     * 
     * @param fieldDef
     *            the field
     */
    public void addField(FieldDef fieldDef)
    {
        if (_fields == null)
            _fields = new ListHashMap<String, FieldDef>();
        
        _fields.put(fieldDef.getName(), fieldDef);
        
        reset();
    }
    
    /**
     * Adds the field at the index.
     * 
     * @param fieldDef
     *            the field
     * @param index
     *            the index
     */
    public void addField(FieldDef fieldDef, int index)
    {
        if (_fields == null)
            _fields = new ListHashMap<String, FieldDef>();
        
        _fields.put(fieldDef.getName(), fieldDef, index);
        
        reset();
    }
    
    /**
     * Adds the record.
     * 
     * @param record
     *            the record
     */
    public boolean addRecord(DataSetRecord record)
    {
        if (_data == null)
            _data = new DataSetData();
        
        if (!filter(record))
            return false;
        
        _data.add(record);
        
        updateIndexOnAdd(record);
        
        return true;
    }
    
    /**
     * Adds the record at the row.
     * 
     * @param record
     *            the record
     * @param row
     *            the row
     * @return true if record was added           
     */
    public boolean addRecord(DataSetRecord record, int row)
    {
        if (_data == null)
            _data = new DataSetData();
        
        if (!filter(record))
            return false;
        
        _data.add(row, record);
        
        return true;
    }
    
    /**
     * Removes all fields and data.
     */
    public void clear()
    {
        if (_fields != null)
            _fields = null;
        clearData();
        
        if (_dataSetIndex != null)
            _dataSetIndex.clear();
        
        reset();
    }
    
    /**
     * Clears data.
     */
    public void clearData()
    {
        if (_data != null)
            _data = null;
    }
    
    /**
     * Copy this data set.
     * 
     * @return copy of the data set
     */
    public DataSet copy()
    {
        DataSet dataSet = new DataSet();
        
        dataSet._name = _name;
        dataSet._ownerName = _ownerName;
        dataSet._data = _data;
        dataSet._fields = _fields;
        dataSet._keyName = _keyName;
        dataSet._keyFields = _keyFields;
        dataSet._displayKeyField = _displayKeyField;
        dataSet._encode = _encode;
        dataSet._driver = _driver;
        dataSet._tableName = _tableName;
        dataSet._connection = _connection;
        dataSet._variables = _variables;
        dataSet._hasLargeObj = _hasLargeObj;
        dataSet._displayFunctions = _displayFunctions;
        dataSet._dataSetIndex = _dataSetIndex;
        _indexFields = null;
        
        return dataSet;
    }
    
    public void copyPropertiesFrom(DataSet dataSet)
    {
        _name = dataSet.getName();
        _ownerName = dataSet.getOwnerName();
        _keyName = null;
        _keyFields = null;
        _encode = dataSet.isEncode();
        _driver = dataSet.getDriver();
        _tableName = dataSet.getTableName();
        _connection = dataSet.getConnection();
        _variables = dataSet.getVariables();
        _hasLargeObj = null;
        _displayFunctions = null;
        _dataSetIndex = null;
        _indexFields = null;
    }
    
    /**
     * Decodes value of the field using Base64 algorithm. Clobs and blobs
     * decoded by default but CHAR fields must have a flag "encode" set to
     * <code>true<code>.
     * If field is not CLOB, BLOB or encoded flavor of CHAR  returns given fieldValue.
     * 
     * @param fieldDef
     *            the field
     * @param fieldValue
     *            the field value
     * @param driver
     *            the driver
     * @param params
     *            the parameters
     * @return the object
     * @throws Exception
     *             in case of any error
     */
    public Object decode(FieldDef fieldDef, String fieldValue, Driver driver,
            Map<String, String> params)
        throws Exception
    {
        return decode(fieldDef, fieldValue, driver, params, true);
    }
    
    /**
     * Decodes value of the field using Base64 algorithm. Clobs and blobs
     * decoded by default but CHAR fields must have a flag "encode" set to
     * <code>true<code>.
     * If field is not CLOB, BLOB or encoded flavor of CHAR  returns given fieldValue.
     * 
     * @param fieldDef
     *            the field
     * @param fieldValue
     *            the field value
     * @param driver
     *            the driver
     * @param params
     *            the parameters
     * @param decode
     *            the decode. If false does not perform decoding
     * @return the object
     * @throws Exception
     *             in case of any error
     */
    public Object decode(FieldDef fieldDef, String fieldValue, Driver driver,
            Map<String, String> params, boolean decode)
        throws Exception
    {
        if (fieldValue == null)
            return null;
        
        driver = driver != null ? driver : getDriver();
        
        Object result;
        
        if (decode
                && _encode
                && (SqlUtils.isLargeObject(fieldDef.getSqlDataType()) || (isFieldEncoded(fieldDef) && SqlUtils
                        .isChar(fieldDef.getSqlDataType()))))
            result = decode(fieldValue, fieldDef.getSqlDataType());
        else
            result = SqlUtils.storageValue2Value(fieldDef.getSqlDataType(),
                    fieldValue, params);
        
        return result;
    }
    
    /**
     * Decodes value of the field using Base64 algorithm. Only CLOB, BLOB and
     * flavors of CHAR fields can be decoded.
     * 
     * @param value
     *            The field value
     * @param fType
     *            The field type. Belongs to <code>java.sql.Types</code>
     * @return the object
     * @throws Exception
     *             in case of any error
     */
    private Object decode(String value, int fType)
        throws Exception
    {
        if (value == null)
            return value;
        
        if (SqlUtils.isClob(fType) || SqlUtils.isChar(fType))
            return Base64.decodeToObject(value);
        else
            return Base64.decode(value);
    }
    
    /**
     * Deletes record at the row.
     * 
     * @param row
     *            the row
     *            
     * @return deleted record            
     */
    public DataSetRecord deleteRecord(int row)
    {
        if (_data == null)
            return null;
        
        DataSetRecord record = _data.delete(row);
        
        if (record != null)
            updateIndexOnDelete(record);
        
        return record;
    }
    
    /**
     * Encodes value of the field using Base64 algorithm. Clobs and blobs
     * encoded by default but flavors of CHAR fields must have a flag "encode"
     * set to <code>true<code>.
     * 
     * @param fieldDef
     *            the field
     * @param fieldValue
     *            The field value
     * @param driver
     *            the driver
     * @param params
     *            the parameters
     * @return the string
     */
    public String encode(FieldDef fieldDef, Object fieldValue, Driver driver,
            Map<String, String> params)
    {
        return encode(fieldDef, fieldValue, driver, params, true);
    }
    
    /**
     * Encodes value of the field using Base64 algorithm. Clobs and blobs
     * encoded by default but flavors of CHAR fields must have a flag "encode"
     * set to <code>true<code>.
     * 
     * @param fieldDef
     *            the field
     * @param fieldValue
     *            the field value
     * @param driver
     *            the driver
     * @param params
     *            the parameters
     * @param encode
     *            the encode. If false does not perform encoding
     * @return the string
     */
    public String encode(FieldDef fieldDef, Object fieldValue, Driver driver,
            Map<String, String> params, boolean encode)
    {
        String result;
        
        driver = driver != null ? driver : getDriver();
        
        Object value = driver.value2StorageValue(fieldDef.getSqlDataType(),
                fieldValue, params);
        
        if (value == null)
            return null;
        
        if (encode
                && _encode
                && (SqlUtils.isLargeObject(fieldDef.getSqlDataType()) || (isFieldEncoded(fieldDef) && SqlUtils
                        .isChar(fieldDef.getSqlDataType()))))
            result = encode(value, fieldDef.getSqlDataType());
        else
            result = value.toString();
        
        return result;
    }
    
    /**
     * Encodes value of the field using Base64 algorithm. Only CLOB, BLOB and
     * flavors of CHAR fields can be encoded.
     * 
     * @param value
     *            The field value
     * @param fType
     *            The field type. Belongs to <code>java.sql.Types</code>
     * @return the string
     */
    private String encode(Object value, int fType)
    {
        if (value == null)
            return null;
        
        if (SqlUtils.isClob(fType) || SqlUtils.isChar(fType))
            return Base64.encodeObject((Serializable)value, Base64.GZIP
                    | Base64.DONT_BREAK_LINES);
        else
            return Base64.encodeBytes((byte[])value, Base64.GZIP
                    | Base64.DONT_BREAK_LINES);
    }
    
    /**
     * Indicates whether some other object is "equal to" this one.
     * 
     * @param o
     *            The object to compare with
     * 
     * @return true, if successful
     */
    @Override
    public boolean equals(Object o)
    {
        if (o == null || !(o instanceof DataSet))
            return false;
        
        DataSet dataSet = (DataSet)o;
        
        if (_fields == null && dataSet.getFields() != null || _fields != null
                && dataSet.getFields() == null
                || _fields.size() != dataSet.getFields().size()
                || _data == null && dataSet.getData() != null || _data != null
                && dataSet.getData() == null
                || _data.size() != dataSet.getData().size())
            return false;
        
        for (int row = 0; row < _data.size(); row++)
        {
            DataSetRecord record = _data.get(row);
            DataSetRecord compareToRecord = dataSet.getData().get(row);
            
            for (int col = 0; col < _fields.size(); col++)
            {
                Object value = record.get(col);
                Object compareToValue = compareToRecord.get(col);
                
                if (value == null && compareToValue == null)
                    continue;
                
                if (value != null && compareToValue == null)
                    return false;
                
                if (value == null && compareToValue != null)
                    return false;
                
                if (value.getClass()
                        .isAssignableFrom(compareToValue.getClass()))
                {
                    if (!value.equals(compareToValue))
                        return false;
                }
                else if (!value.toString().equals(compareToValue.toString()))
                    return false;
                
            }
        }
        
        return true;
    }
    
    /**
     * Filters record using expression.
     *
     * @param record the record
     * @return true if record can be added
     */
    public boolean filter(DataSetRecord record)
    {
        if (_script == null)
            return true;
        
        if (_nonCaseSensitiveFields == null)
            _nonCaseSensitiveFields = getNonCaseSensitiveFields();
        
        Bindings bindings = null;
        
        if (!_script.hasCompiledCode(null, "filter"))
            try
            {
                if (!_condVars.isEmpty() && _fields != null)
                    for (String name : _condVars.keySet())
                        if (_fields.containsKey(name))
                        {
                            String newName = name.replaceAll(" ", "_");
                            
                            _filter = _filter.replaceAll("\"" + name + "\"",
                                    newName);
                        }
                
                _script.compile(null, "filter", Script.any2Js(_filter),
                        "JavaScript");
            }
            catch (Exception ex)
            {
                throw new RuntimeException(ex);
            }
        
        try
        {
            if (_condVars.size() > 0)
            {
                bindings = _script.getBindings(null, "JavaScript");
                
                for (String name : _condVars.keySet())
                {
                    FieldDef field = _nonCaseSensitiveFields.get(name);
                    
                    if (field != null)
                    {
                        int col = getFieldIndex(field.getName());
                        
                        bindings.put(name.replaceAll(" ", "_"), record.get(col));
                    }
                }
            }
            
            Object ret = _script.eval(null, bindings, "filter", "JavaScript");
            
            return Boolean.TRUE.equals(ret);
        }
        catch (Exception ex)
        {
            throw new RuntimeException(ex);
        }
    }
    
    /**
     * Gets the connection.
     * 
     * @return the connection
     */
    public Connection getConnection()
    {
        return _connection;
    }
    
    /**
     * Gets the current record as a "cursor" using given driver.
     * 
     * @param driver
     *            the driver
     * @return the current record
     */
    public DataSetRecord getCursorRecord(Driver driver)
    {
        int count = getFieldCount();
        
        if (count == 0)
            return null;
        
        driver = driver != null ? driver : getDriver();
        
        DataSetRecord record = new DataSetRecord();
        
        for (int i = 0; i < count; i++)
            record.add(driver.getCursorRecAccessSql(getFieldDef(i).getName()));
        
        return record;
    }
    
    /**
     * Gets the data.
     * 
     * @return the data
     */
    public DataSetData getData()
    {
        return _data;
    }
    
    /**
     * Gets the index
     * @return the index
     */
    public Map<String, DataSetRecord> getDataSetIndex()
    {
        return _dataSetIndex;
    }
    
    /**
     * Gets the display value of the field for the field.
     * 
     * @param value
     *            the value
     * @param field
     *            the field
     * @return the display field value
     */
    public Object getDisplayFieldValue(Object value, FieldDef field)
    {
        if (_displayFunctions == null || _displayFunctions.size() == 0
                || field == null)
            return value;
        
        TypedKeyValue<String, String> fnc = _displayFunctions.get(field
                .getName());
        
        if (fnc == null)
            return value;
        
        return Utils.callStatic(fnc.getKey(), fnc.getValue(),
                new Class<?>[] {Object.class}, value);
        
    }
    
    /**
     * Gets the display value of the field from the record by the index of the
     * field.
     * 
     * @param value
     *            the value
     * @param index
     *            the index of the field
     * @return the display field value
     */
    public Object getDisplayFieldValue(Object value, int index)
    {
        return getDisplayFieldValue(value, getFieldDef(index));
    }
    
    /**
     * Gets the display value of the field from the record by the name of the
     * field.
     * 
     * @param value
     *            the value
     * @param name
     *            the name
     * @return the display field value
     */
    public Object getDisplayFieldValue(Object value, String name)
    {
        return getDisplayFieldValue(value, getFieldDef(name));
    }
    
    /**
     * Gets the "display name" of the key field.
     * 
     * @return the "display name" of the key field.
     */
    public String getDisplayKeyField()
    {
        return _displayKeyField;
    }
    
    /**
     * Gets the driver.
     * 
     * @return the driver
     */
    public Driver getDriver()
    {
        return _driver;
    }
    
    /**
     * Gets the field's attribute by given name. Any field can have an any
     * number of the additional attributes stored as a key\value pairs.
     * 
     * @param fieldDef
     *            the field
     * @param name
     *            the name of the attribute
     * @return the attribute value
     */
    public String getFieldAttr(FieldDef fieldDef, String name)
    {
        if (_variables == null || _variables.size() == 0)
            return null;
        
        Variable var = _variables.get(fieldDef.getName());
        
        if (var == null || var.getAttrs() == null)
            return null;
        
        return var.getAttrs().get(name);
    }
    
    /**
     * Gets the number of fields.
     * 
     * @return the number of fields
     */
    public int getFieldCount()
    {
        if (_fields == null)
            return 0;
        
        return _fields.size();
    }
    
    /**
     * Gets the field by index.
     * 
     * @param pos
     *            The index of the field
     * 
     * @return the field
     */
    public FieldDef getFieldDef(int pos)
    {
        if (_fields == null)
            return null;
        
        return _fields.get(pos);
    }
    
    /**
     * Gets the field by name.
     * 
     * @param name
     *            The name of the field
     * 
     * @return the field
     */
    public FieldDef getFieldDef(String name)
    {
        if (_fields == null)
            return null;
        
        return _fields.get(name);
    }
    
    /**
     * Gets the index of the field by name of the field.
     * 
     * @param name
     *            the name of the field
     * @return the index of the field
     */
    public int getFieldIndex(String name)
    {
        if (name == null || _fields == null)
            return -1;
        
        return _fields.indexOf(getFieldDef(name));
    }
    
    /**
     * Gets the fields.
     * 
     * @return the fields
     */
    public ListHashMap<String, FieldDef> getFields()
    {
        return _fields;
    }
    
    /**
     * Gets the value of the field from the record by the index of the field.
     * 
     * @param record
     *            The record
     * @param index
     *            The index of the field
     * @return the value of the field
     */
    public Object getFieldValue(DataSetRecord record, int index)
    {
        if (record == null)
            return null;
        
        return index >= 0 && index < record.size() ? record.get(index) : null;
    }
    
    /**
     * Gets the value of the field from the record by the name of the field.
     * 
     * @param record
     *            The record
     * @param name
     *            The name of the field
     * @return the value of the field
     */
    public Object getFieldValue(DataSetRecord record, String name)
    {
        if (record == null || name == null || _fields == null)
            return null;
        
        return getFieldValue(record, getFieldIndex(name));
    }
    
    /**
     * Gets the value of the field from the record <code>row</code> by the index
     * of the <code>col</code>.
     * 
     * @param row
     *            the row
     * @param col
     *            the column
     * @return the value of the field
     */
    public Object getFieldValue(int row, int col)
    {
        if (_data == null || _fields == null)
            return null;
        
        return getFieldValue(getRecord(row), col);
    }
    
    /**
     * Gets the data set file name using given folder and extension.
     * 
     * @param dataLocation
     *            The path to the file
     * @param ext
     *            The extension
     * 
     * @return the full file name
     */
    public String getFileName(String dataLocation, String ext)
    {
        return FileUtils.getFullFileName(dataLocation, _name, ext, false);
    }
    
    /**
     * Gets the names of the key fields.
     * 
     * @return the names of the key fields.
     */
    public String getKeyFields()
    {
        return _keyFields;
    }
    
    /**
     * Gets the name.
     * 
     * @return the name
     */
    public String getName()
    {
        return _name;
    }
    
    public TreeMap<String, FieldDef> getNonCaseSensitiveFields()
    {
        TreeMap<String, FieldDef> fields = new TreeMap<String, FieldDef>(
                String.CASE_INSENSITIVE_ORDER);
        
        if (_fields == null || _fields.size() == 0)
            return fields;
        
        for (FieldDef field : _fields.getList())
        {
            fields.put(field.getName(), field);
        }
        
        return fields;
    }
    
    /**
     * Gets the object name which is either data set name or table name.
     * 
     * <p>
     * <code>Utils.isNothing(getTableName()) ? getName() : getTableName();</code>
     * 
     * @return the object name
     */
    public String getObjectName()
    {
        return Utils.isNothing(getTableName()) ? getName() : getTableName();
    }
    
    /**
     * Gets the owner name.
     * 
     * @return the owner name
     */
    public String getOwnerName()
    {
        return _ownerName;
    }
    
    /**
     * Gets the record using given index of the record.
     * 
     * @param row
     *            The index of the record
     * @return the record
     */
    public DataSetRecord getRecord(int row)
    {
        if (_data == null)
            return null;
        
        return _data.get(row);
    }
    
    /**
     * Get record by key
     * @param key the key
     * @return the record
     */
    public DataSetRecord getRecord(String key)
    {
        if (_dataSetIndex == null || _dataSetIndex.size() == 0
                || Utils.isNothing(key) || Utils.isNothing(_keyFields))
            return null;
        
        return _dataSetIndex.get(key);
    }
    
    /**
     * Gets the number of records.
     * 
     * @return the number of records
     */
    public int getRecordCount()
    {
        return _data != null ? _data.size() : 0;
    }
    
    /**
     * Gets the table name.
     * 
     * @return the table name
     */
    public String getTableName()
    {
        return _tableName;
    }
    
    /**
     * Gets the variables.
     * 
     * @return the variables
     */
    public Map<String, Variable> getVariables()
    {
        return _variables;
    }
    
    /**
     * Checks if data set has CLOBs or BLOBs.
     * 
     * @return true, if successful
     */
    public boolean hasLargeObjects()
    {
        if (_hasLargeObj != null)
            return _hasLargeObj;
        
        if (getFieldCount() == 0)
            return false;
        
        int fCount = getFieldCount();
        
        for (int col = 0; col < fCount; col++)
        {
            if (getFieldDef(col).isLargeObject())
            {
                _hasLargeObj = true;
                
                return true;
            }
        }
        
        _hasLargeObj = false;
        
        return false;
    }
    
    /**
     * Checks if data set is empty.
     * 
     * @return true, if data set is empty.
     */
    public boolean isEmpty()
    {
        return getData() == null || getData().isEmpty();
    }
    
    /**
     * Checks if encoded flag set for the data set. The default value is
     * <code>true</code>.
     * 
     * @return true, if encoded
     */
    public boolean isEncode()
    {
        return _encode;
    }
    
    /**
     * Checks if field is encoded.
     * 
     * @param fieldDef
     *            the field
     * @return true, if field is encoded.
     */
    public boolean isFieldEncoded(FieldDef fieldDef)
    {
        if (fieldDef.isEncode())
            return true;
        
        String value = getFieldAttr(fieldDef, ENCODE_ATTR);
        
        if (value == null)
            return false;
        
        return Utils.str2Boolean(value, Boolean.FALSE);
    }
    
    /**
     * Removes the field.
     * 
     * @param fieldDef
     *            the field
     */
    public void removeField(FieldDef fieldDef)
    {
        if (_fields == null)
            return;
        
        _fields.remove(fieldDef.getName());
        
        reset();
    }
    
    /**
     * Clear all internal flags.
     */
    private void reset()
    {
        _hasLargeObj = null;
        _indexFields = null;
    }
    
    /**
     * Sets the connection.
     * 
     * @param value
     *            the new connection
     */
    public void setConnection(Connection value)
    {
        _connection = value;
    }
    
    /**
     * Sets the data.
     * 
     * @param value
     *            the new data
     */
    public void setData(DataSetData value)
    {
        _data = value;
    }
    
    /**
     * Sets the index..
     *  
     * @param value the index
     */
    public void setDataSetIndex(Map<String, DataSetRecord> value)
    {
        _dataSetIndex = value;
    }
    
    /**
     * Sets the display functions.
     * 
     * @param functions
     *            the functions
     */
    public void setDisplayFunctions(
            Map<String, TypedKeyValue<String, String>> functions)
    {
        _displayFunctions = functions;
    }
    
    /**
     * Sets the "display name" for the key field.
     * 
     * @param value
     *            the new the "display name" for the key field.
     */
    public void setDisplayKeyField(String value)
    {
        _displayKeyField = value;
    }
    
    /**
     * Sets the driver.
     * 
     * @param value
     *            the new driver
     */
    public void setDriver(Driver value)
    {
        _driver = value;
    }
    
    /**
     * Sets the encoded flag for the data set. The default value is
     * <code>true</code>.
     * 
     * @param value
     *            the new encode
     */
    public void setEncode(boolean value)
    {
        _encode = value;
    }
    
    /**
     * Replaces current field defined by field.getName() on the given field.
     * 
     * @param fieldDef
     *            the new field
     */
    public void setField(FieldDef fieldDef)
    {
        if (_fields == null)
            _fields = new ListHashMap<String, FieldDef>();
        
        _fields.set(fieldDef.getName(), fieldDef, true);
    }
    
    /**
     * Sets the attribute of the field.
     * 
     * @param fieldDef
     *            the field
     * @param name
     *            the attribute's name
     * @param value
     *            the attribute's value
     */
    public void setFieldAttr(FieldDef fieldDef, String name, String value)
    {
        if (_variables == null)
            _variables = new HashMap<String, Variable>();
        
        Variable var = _variables.get(fieldDef.getName());
        
        if (var == null)
            var = new Variable();
        
        if (var.getAttrs() == null)
            var.setAttrs(new HashMap<String, String>());
        
        var.getAttrs().put(name, value);
        
        _variables.put(fieldDef.getName(), var);
    }
    
    /**
     * Sets the fields.
     * 
     * @param value
     *            the value
     */
    public void setFields(ListHashMap<String, FieldDef> value)
    {
        _fields = value;
        
        reset();
    }
    
    /**
     * Sets the field value.
     * 
     * @param record
     *            the record
     * @param index
     *            the index of the field
     * @param value
     *            the value
     */
    public void setFieldValue(DataSetRecord record, int index, Object value)
    {
        if (record == null)
            return;
        
        record.set(index, value);
    }
    
    /**
     * Sets the field value.
     * 
     * @param record
     *            the record
     * @param name
     *            the name of the field
     * @param value
     *            the value
     */
    public void setFieldValue(DataSetRecord record, String name, Object value)
    {
        if (record == null || name == null || _fields == null)
            return;
        
        setFieldValue(record, getFieldIndex(name), value);
    }
    
    /**
     * Sets the field value.
     * 
     * @param row
     *            the index of the record
     * @param col
     *            the index of the field
     * @param value
     *            the value
     */
    public void setFieldValue(int row, int col, Object value)
    {
        DataSetRecord record = getRecord(row);
        
        if (record == null)
            return;
        
        record.set(col, value);
    }
    
    /**
     * Sets the field value.
     * 
     * @param row
     *            the index of the record
     * @param name
     *            the name of the field
     * @param value
     *            the value
     */
    public void setFieldValue(int row, String name, Object value)
    {
        DataSetRecord record = getRecord(row);
        
        if (record == null)
            return;
        
        setFieldValue(record, getFieldIndex(name), value);
    }
    
    /** 
     * Sets the filter.
     * 
     * <pre>
     * Example: col1 = 'test' and col2 = 'xyz'
     * </pre>
     * 
     * @param filter the filter
     */
    public void setFilter(String filter)
        throws Exception
    {
        if (Utils.isNothing(filter))
        {
            _script = null;
            _nonCaseSensitiveFields = null;
            _condVars = null;
            _filter = null;
            
            return;
        }
        
        _script = new Script();
        
        filter = Script.sql2Java(filter);
        
        _condVars = Script.getVariables(filter);
        
        _filter = Script.any2Js(filter);
    }
    
    /**
     * Sets the names of the key fields.
     * 
     * @param value
     *            The new names of the key fields
     */
    public void setKeyFields(String value)
    {
        _keyFields = value;
        
        if (!Utils.isNothing(_keyFields))
        {
            if (_dataSetIndex == null)
                _dataSetIndex = new HashMap<String, DataSetRecord>();
            else
                _dataSetIndex.clear();
        }
    }
    
    /**
     * Sets the name.
     * 
     * @param value
     *            the new name
     */
    public void setName(String value)
    {
        _name = value;
    }
    
    /**
     * Sets the owner name.
     * 
     * @param value
     *            the new owner name
     */
    public void setOwnerName(String value)
    {
        _ownerName = value;
    }
    
    /**
     * Sets the record.
     * 
     * @param record
     *            the record
     * @param row
     *            the index of the record
     */
    public void setRecord(DataSetRecord record, int row)
    {
        if (_data == null)
            _data = new DataSetData();
        
        _data.set(row, record);
    }
    
    /**
     * Sets the table name.
     * 
     * @param value
     *            the new table name
     */
    public void setTableName(String value)
    {
        _tableName = value;
    }
    
    /**
     * Sets the variables.
     * 
     * @param value
     *            the value
     */
    public void setVariables(Map<String, Variable> value)
    {
        _variables = value;
    }
    
    /** Sorts data using given order by list with field indexes.
     *  
     * @param orderBy the order by list
    */
    public void sortByFieldIndex(
            final List<TypedKeyValue<Integer, SortType>> orderBy)
    {
        if (_fields == null || _fields.size() == 0 || _data == null
                || _data.size() == 0 || orderBy == null || orderBy.size() == 0)
            return;
        
        _data.sort(orderBy);
    }
    
    /** Sorts data using given order by list with field names.
     *  
     * @param orderBy the order by list
    */
    public void sortByFieldName(
            final List<TypedKeyValue<String, SortType>> orderBy)
    {
        if (_fields == null || _fields.size() == 0 || _data == null
                || _data.size() == 0 || orderBy == null || orderBy.size() == 0)
            return;
        
        List<TypedKeyValue<Integer, SortType>> indexOrderBy = new ArrayList<TypedKeyValue<Integer, SortType>>();
        
        TreeMap<String, FieldDef> nonCaseSens = getNonCaseSensitiveFields();
        
        for (TypedKeyValue<String, SortType> item : orderBy)
        {
            
            FieldDef field = nonCaseSens.get(item.getKey());
            
            int index = _fields.indexOf(field);
            
            if (index >= 0)
                indexOrderBy.add(new TypedKeyValue<Integer, SortType>(index,
                        item.getValue()));
        }
        
        if (indexOrderBy.size() == 0)
            return;
        
        _data.sort(indexOrderBy);
    }
    
    /** Sorts data using given "order by" string.
     *  
     *  <br>
     *  Example: first desc,last
     *  
     * @param orderBy the "order by" string
    */
    
    public void sortByString(String orderBy)
    {
        if (_fields == null || _fields.size() == 0 || _data == null
                || _data.size() == 0 || Utils.isNothing(orderBy))
            return;
        
        String[] tokens = orderBy.split(",", -1);
        
        List<TypedKeyValue<Integer, SortType>> indexOrderBy = new ArrayList<TypedKeyValue<Integer, SortType>>();
        
        TreeMap<String, FieldDef> nonCaseSens = getNonCaseSensitiveFields();
        
        for (String token : tokens)
        {
            String[] sort = token.trim().split(" ", -1);
            
            if (sort.length > 2)
                continue;
            
            FieldDef field = nonCaseSens.get(sort[0]);
            
            int index = _fields.indexOf(field);
            
            if (index < 0)
                continue;
            
            if (sort.length == 1)
            {
                indexOrderBy.add(new TypedKeyValue<Integer, SortType>(index,
                        DataSetData.SortType.ASC));
                
                continue;
            }
            
            if ("ASC".equalsIgnoreCase(sort[1]))
            {
                indexOrderBy.add(new TypedKeyValue<Integer, SortType>(index,
                        DataSetData.SortType.ASC));
                
            }
            else if ("DESC".equalsIgnoreCase(sort[1]))
            {
                indexOrderBy.add(new TypedKeyValue<Integer, SortType>(index,
                        DataSetData.SortType.DESC));
            }
        }
        
        if (indexOrderBy.size() == 0)
            return;
        
        _data.sort(indexOrderBy);
        
    }
    
    /**
     * Updates index on "add record" event.
     * 
     * @param currentRow
     *            the record
     */
    public void updateIndexOnAdd(DataSetRecord currentRow)
    {
        if (currentRow == null || Utils.isNothing(_keyFields))
            return;
        
        if (_indexFields == null)
            _indexFields = CommonEtlUtils.getKeyFields(_keyFields, _fields);
        
        if (_indexFields.size() == 0)
            return;
        
        String key = CommonEtlUtils.getKey(this, currentRow, _indexFields,
                false, true);
        
        if (Utils.isNothing(key))
            return;
        
        _dataSetIndex.put(key, currentRow);
    }
    
    /**
     * Updates index on "delete record" event.
     * 
     * @param currentRow
     *            the record
     */
    public void updateIndexOnDelete(DataSetRecord currentRow)
    {
        if (currentRow == null || Utils.isNothing(_keyFields))
            return;
        
        if (_indexFields == null)
            _indexFields = CommonEtlUtils.getKeyFields(_keyFields, _fields);
        
        if (_indexFields.size() == 0)
            return;
        
        String key = CommonEtlUtils.getKey(this, currentRow, _indexFields,
                false, true);
        
        if (Utils.isNothing(key))
            return;
        
        _dataSetIndex.remove(key);
    }
    
}
